{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-09T21:33:13.726894Z",
     "start_time": "2017-05-09T21:33:10.549232Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.layers import Input, Dense, Lambda, Layer\n",
    "from keras.models import Model\n",
    "from keras.layers.core import Dropout\n",
    "from keras import regularizers\n",
    "import keras\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from keras import backend as K\n",
    "from keras import metrics\n",
    "from collections import namedtuple\n",
    "pd.set_option(\"display.max_rows\",35)\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-09T21:33:15.050351Z",
     "start_time": "2017-05-09T21:33:13.730097Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kdd_train_2labels = pd.read_pickle(\"dataset/kdd_train_2labels.pkl\")\n",
    "kdd_test_2labels = pd.read_pickle(\"dataset/kdd_test_2labels.pkl\")\n",
    "\n",
    "#y_train_labels = pd.read_pickle(\"dataset/kdd_train_2labels_y.pkl\")\n",
    "#y_train_labels = pd.read_pickle(\"dataset/kdd_train_2labels.pkl\")\n",
    "#y_test_labels = pd.read_pickle(\"dataset/kdd_test_2labels_y.pkl\")\n",
    "\n",
    "output_columns_2labels = ['is_Attack','is_Normal']\n",
    "\n",
    "from sklearn import model_selection as ms\n",
    "from sklearn import preprocessing as pp\n",
    "\n",
    "x_input = kdd_train_2labels.drop(output_columns_2labels, axis = 1)\n",
    "y_output = kdd_train_2labels.loc[:,output_columns_2labels]\n",
    "\n",
    "ss = pp.StandardScaler()\n",
    "x_input = ss.fit_transform(x_input)\n",
    "\n",
    "#le = pp.LabelEncoder()\n",
    "#y_train = le.fit_transform(y_train_labels).reshape(-1, 1)\n",
    "#y_test = le.transform(y_test_labels).reshape(-1, 1)\n",
    "\n",
    "y_train = kdd_train_2labels.loc[:,output_columns_2labels].values\n",
    "\n",
    "x_train, x_valid, y_train, y_valid = ms.train_test_split(x_input, \n",
    "                              y_train, \n",
    "                              test_size=0.1)\n",
    "#x_valid, x_test, y_valid, y_test = ms.train_test_split(x_valid, y_valid, test_size = 0.4)\n",
    "\n",
    "x_test = kdd_test_2labels.drop(output_columns_2labels, axis = 1)\n",
    "y_test = kdd_test_2labels.loc[:,output_columns_2labels].values\n",
    "\n",
    "x_test = ss.transform(x_test)\n",
    "\n",
    "#x_train = np.hstack((x_train, y_train))\n",
    "#x_valid = np.hstack((x_valid, y_valid))\n",
    "\n",
    "#x_test = np.hstack((x_test, np.random.normal(loc = 0, scale = 0.01, size = y_test.shape)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2017-05-09T21:33:11.992Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "input_dim = 122\n",
    "intermediate_dim = 10\n",
    "latent_dim = 32\n",
    "batch_size = 1409\n",
    "hidden_layers = 8\n",
    "classes = 2\n",
    "drop_prob = 0.1\n",
    "\n",
    "class Train:\n",
    "    def build_vae_model():\n",
    "        Train.x = Input(shape=(input_dim,))\n",
    "        \n",
    "        hidden_encoder = Train.x\n",
    "        for i in range(hidden_layers):\n",
    "            hidden_encoder = Dense(intermediate_dim, activation='relu', \n",
    "                      kernel_regularizer=keras.regularizers.l2(0.0001),\n",
    "                      activity_regularizer=keras.regularizers.l1(0.0001))(hidden_encoder)\n",
    "            \n",
    "            hidden_encoder = Dropout(rate=drop_prob)(hidden_encoder)\n",
    "\n",
    "        Train.mean_encoder = Dense(latent_dim, activation=None)(hidden_encoder)\n",
    "        Train.logvar_encoder = Dense(latent_dim, activation=None)(hidden_encoder)\n",
    "\n",
    "        def get_distrib(args):\n",
    "\n",
    "            m_e, l_e = args\n",
    "\n",
    "            # Sample epsilon\n",
    "            epsilon = np.random.normal(loc=0.0, scale=0.05, size = (batch_size, latent_dim))\n",
    "\n",
    "            # Sample latent variable\n",
    "            z = m_e + K.exp(l_e / 2) * epsilon\n",
    "            return z\n",
    "\n",
    "        z = Lambda(get_distrib,name='z_dist')([Train.mean_encoder, Train.logvar_encoder])\n",
    "\n",
    "        hidden_decoder = z\n",
    "        for i in range(hidden_layers):\n",
    "            hidden_decoder = Dense(intermediate_dim, activation=\"relu\", \n",
    "                      kernel_regularizer=keras.regularizers.l2(0.0001),\n",
    "                      activity_regularizer=keras.regularizers.l1(0.0001))(hidden_decoder)\n",
    "            hidden_decoder = Dropout(rate=drop_prob)(hidden_decoder)\n",
    "\n",
    "        Train.x_ = Dense(input_dim, activation=None, name='vae_output')(hidden_decoder)\n",
    "        \n",
    "        #Train.z_ = Input(shape=(latent_dim,))\n",
    "        hidden_y = Dense(latent_dim, activation='relu', name='softmax_hidden')(z)\n",
    "        Train.y = Dense(classes, activation='softmax', name='softmax_output')(hidden_y)\n",
    "        \n",
    "Train.build_vae_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2017-05-09T21:33:11.997Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      " Current Layer Attributes - epochs:50 hidden layers:2 features count:4\n",
      "Train on 112720 samples, validate on 22544 samples\n",
      "Epoch 1/50\n",
      "112720/112720 [==============================] - 3s - loss: 0.9888 - acc: 0.5345 - val_loss: 0.8699 - val_acc: 0.4308\n",
      "Epoch 2/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7346 - acc: 0.5353 - val_loss: 0.8078 - val_acc: 0.4308\n",
      "Epoch 3/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7164 - acc: 0.5353 - val_loss: 0.7868 - val_acc: 0.4308\n",
      "Epoch 4/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7110 - acc: 0.5353 - val_loss: 0.7763 - val_acc: 0.4308\n",
      "Epoch 5/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7085 - acc: 0.5353 - val_loss: 0.7698 - val_acc: 0.4308\n",
      "Epoch 6/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7072 - acc: 0.5353 - val_loss: 0.7657 - val_acc: 0.4308\n",
      "Epoch 7/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7062 - acc: 0.5353 - val_loss: 0.7628 - val_acc: 0.4308\n",
      "Epoch 8/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7053 - acc: 0.5353 - val_loss: 0.7603 - val_acc: 0.4308\n",
      "Epoch 9/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7046 - acc: 0.5353 - val_loss: 0.7590 - val_acc: 0.4308\n",
      "Epoch 10/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7045 - acc: 0.5353 - val_loss: 0.7579 - val_acc: 0.4308\n",
      "Epoch 11/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7041 - acc: 0.5353 - val_loss: 0.7569 - val_acc: 0.4308\n",
      "Epoch 12/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7036 - acc: 0.5353 - val_loss: 0.7559 - val_acc: 0.4308\n",
      "Epoch 13/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7034 - acc: 0.5353 - val_loss: 0.7553 - val_acc: 0.4308\n",
      "Epoch 14/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7036 - acc: 0.5353 - val_loss: 0.7545 - val_acc: 0.4308\n",
      "Epoch 15/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7031 - acc: 0.5353 - val_loss: 0.7537 - val_acc: 0.4308\n",
      "Epoch 16/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7028 - acc: 0.5353 - val_loss: 0.7531 - val_acc: 0.4308\n",
      "Epoch 17/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7028 - acc: 0.5353 - val_loss: 0.7525 - val_acc: 0.4308\n",
      "Epoch 18/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7022 - acc: 0.5353 - val_loss: 0.7520 - val_acc: 0.4308\n",
      "Epoch 19/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7024 - acc: 0.5353 - val_loss: 0.7516 - val_acc: 0.4308\n",
      "Epoch 20/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7023 - acc: 0.5353 - val_loss: 0.7512 - val_acc: 0.4308\n",
      "Epoch 21/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7021 - acc: 0.5353 - val_loss: 0.7508 - val_acc: 0.4308\n",
      "Epoch 22/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7019 - acc: 0.5353 - val_loss: 0.7503 - val_acc: 0.4308\n",
      "Epoch 23/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7019 - acc: 0.5353 - val_loss: 0.7499 - val_acc: 0.4308\n",
      "Epoch 24/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7016 - acc: 0.5353 - val_loss: 0.7496 - val_acc: 0.4308\n",
      "Epoch 25/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7015 - acc: 0.5353 - val_loss: 0.7492 - val_acc: 0.4308\n",
      "Epoch 26/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7015 - acc: 0.5353 - val_loss: 0.7490 - val_acc: 0.4308\n",
      "Epoch 27/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7013 - acc: 0.5353 - val_loss: 0.7486 - val_acc: 0.4308\n",
      "Epoch 28/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7013 - acc: 0.5353 - val_loss: 0.7483 - val_acc: 0.4308\n",
      "Epoch 29/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7011 - acc: 0.5353 - val_loss: 0.7480 - val_acc: 0.4308\n",
      "Epoch 30/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7011 - acc: 0.5353 - val_loss: 0.7477 - val_acc: 0.4308\n",
      "Epoch 31/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7010 - acc: 0.5353 - val_loss: 0.7474 - val_acc: 0.4308\n",
      "Epoch 32/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7009 - acc: 0.5353 - val_loss: 0.7472 - val_acc: 0.4308\n",
      "Epoch 33/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7007 - acc: 0.5353 - val_loss: 0.7469 - val_acc: 0.4308\n",
      "Epoch 34/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7006 - acc: 0.5353 - val_loss: 0.7467 - val_acc: 0.4308\n",
      "Epoch 35/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7009 - acc: 0.5353 - val_loss: 0.7465 - val_acc: 0.4308\n",
      "Epoch 36/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7008 - acc: 0.5353 - val_loss: 0.7462 - val_acc: 0.4308\n",
      "Epoch 37/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7007 - acc: 0.5353 - val_loss: 0.7460 - val_acc: 0.4308\n",
      "Epoch 38/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7005 - acc: 0.5353 - val_loss: 0.7457 - val_acc: 0.4308\n",
      "Epoch 39/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7004 - acc: 0.5353 - val_loss: 0.7455 - val_acc: 0.4308\n",
      "Epoch 40/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7004 - acc: 0.5353 - val_loss: 0.7452 - val_acc: 0.4308\n",
      "Epoch 41/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7003 - acc: 0.5353 - val_loss: 0.7451 - val_acc: 0.4308\n",
      "Epoch 42/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7001 - acc: 0.5353 - val_loss: 0.7449 - val_acc: 0.4308\n",
      "Epoch 43/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7001 - acc: 0.5353 - val_loss: 0.7446 - val_acc: 0.4308\n",
      "Epoch 44/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7001 - acc: 0.5353 - val_loss: 0.7444 - val_acc: 0.4308\n",
      "Epoch 45/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.7000 - acc: 0.5353 - val_loss: 0.7442 - val_acc: 0.4308\n",
      "Epoch 46/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6999 - acc: 0.5353 - val_loss: 0.7440 - val_acc: 0.4308\n",
      "Epoch 47/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6999 - acc: 0.5353 - val_loss: 0.7439 - val_acc: 0.4308\n",
      "Epoch 48/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6998 - acc: 0.5353 - val_loss: 0.7437 - val_acc: 0.4308\n",
      "Epoch 49/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6999 - acc: 0.5353 - val_loss: 0.7436 - val_acc: 0.4308\n",
      "Epoch 50/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6997 - acc: 0.5353 - val_loss: 0.7433 - val_acc: 0.4308\n",
      "19726/22544 [=========================>....] - ETA: 0s\n",
      " Train Acc: 0.529719665646553, Test Acc: 0.43075762689113617\n",
      " \n",
      " Current Layer Attributes - epochs:50 hidden layers:2 features count:16\n",
      "Train on 112720 samples, validate on 22544 samples\n",
      "Epoch 1/50\n",
      "112720/112720 [==============================] - 3s - loss: 0.6986 - acc: 0.5353 - val_loss: 0.7302 - val_acc: 0.4308\n",
      "Epoch 2/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6972 - acc: 0.5353 - val_loss: 0.7288 - val_acc: 0.4308\n",
      "Epoch 3/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6966 - acc: 0.5353 - val_loss: 0.7271 - val_acc: 0.4308\n",
      "Epoch 4/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6962 - acc: 0.5353 - val_loss: 0.7266 - val_acc: 0.4308\n",
      "Epoch 5/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6961 - acc: 0.5353 - val_loss: 0.7260 - val_acc: 0.4308\n",
      "Epoch 6/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6958 - acc: 0.5353 - val_loss: 0.7252 - val_acc: 0.4308\n",
      "Epoch 7/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6956 - acc: 0.5353 - val_loss: 0.7251 - val_acc: 0.4308\n",
      "Epoch 8/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6955 - acc: 0.5353 - val_loss: 0.7250 - val_acc: 0.4308\n",
      "Epoch 9/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6954 - acc: 0.5353 - val_loss: 0.7245 - val_acc: 0.4308\n",
      "Epoch 10/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6953 - acc: 0.5353 - val_loss: 0.7242 - val_acc: 0.4308\n",
      "Epoch 11/50\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "112720/112720 [==============================] - 1s - loss: 0.6952 - acc: 0.5353 - val_loss: 0.7237 - val_acc: 0.4308\n",
      "Epoch 12/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6951 - acc: 0.5353 - val_loss: 0.7234 - val_acc: 0.4308\n",
      "Epoch 13/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6950 - acc: 0.5353 - val_loss: 0.7231 - val_acc: 0.4308\n",
      "Epoch 14/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6950 - acc: 0.5353 - val_loss: 0.7229 - val_acc: 0.4308\n",
      "Epoch 15/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6949 - acc: 0.5353 - val_loss: 0.7227 - val_acc: 0.4308\n",
      "Epoch 16/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6948 - acc: 0.5353 - val_loss: 0.7226 - val_acc: 0.4308\n",
      "Epoch 17/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6948 - acc: 0.5353 - val_loss: 0.7224 - val_acc: 0.4308\n",
      "Epoch 18/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6948 - acc: 0.5353 - val_loss: 0.7222 - val_acc: 0.4308\n",
      "Epoch 19/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6947 - acc: 0.5353 - val_loss: 0.7220 - val_acc: 0.4308\n",
      "Epoch 20/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6946 - acc: 0.5353 - val_loss: 0.7218 - val_acc: 0.4308\n",
      "Epoch 21/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6945 - acc: 0.5353 - val_loss: 0.7218 - val_acc: 0.4308\n",
      "Epoch 22/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6945 - acc: 0.5353 - val_loss: 0.7216 - val_acc: 0.4308\n",
      "Epoch 23/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6945 - acc: 0.5353 - val_loss: 0.7215 - val_acc: 0.4308\n",
      "Epoch 24/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6945 - acc: 0.5353 - val_loss: 0.7214 - val_acc: 0.4308\n",
      "Epoch 25/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6944 - acc: 0.5353 - val_loss: 0.7212 - val_acc: 0.4308\n",
      "Epoch 26/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6944 - acc: 0.5353 - val_loss: 0.7211 - val_acc: 0.4308\n",
      "Epoch 27/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6944 - acc: 0.5353 - val_loss: 0.7209 - val_acc: 0.4308\n",
      "Epoch 28/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6943 - acc: 0.5353 - val_loss: 0.7208 - val_acc: 0.4308\n",
      "Epoch 29/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6943 - acc: 0.5353 - val_loss: 0.7208 - val_acc: 0.4308\n",
      "Epoch 30/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6943 - acc: 0.5353 - val_loss: 0.7206 - val_acc: 0.4308\n",
      "Epoch 31/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6943 - acc: 0.5353 - val_loss: 0.7205 - val_acc: 0.4308\n",
      "Epoch 32/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6941 - acc: 0.5353 - val_loss: 0.7205 - val_acc: 0.4308\n",
      "Epoch 33/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6942 - acc: 0.5353 - val_loss: 0.7204 - val_acc: 0.4308\n",
      "Epoch 34/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6942 - acc: 0.5353 - val_loss: 0.7203 - val_acc: 0.4308\n",
      "Epoch 35/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6941 - acc: 0.5353 - val_loss: 0.7202 - val_acc: 0.4308\n",
      "Epoch 36/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6941 - acc: 0.5353 - val_loss: 0.7201 - val_acc: 0.4308\n",
      "Epoch 37/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6941 - acc: 0.5353 - val_loss: 0.7201 - val_acc: 0.4308\n",
      "Epoch 38/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6941 - acc: 0.5353 - val_loss: 0.7199 - val_acc: 0.4308\n",
      "Epoch 39/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6940 - acc: 0.5353 - val_loss: 0.7198 - val_acc: 0.4308\n",
      "Epoch 40/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6939 - acc: 0.5353 - val_loss: 0.7197 - val_acc: 0.4308\n",
      "Epoch 41/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6940 - acc: 0.5353 - val_loss: 0.7197 - val_acc: 0.4308\n",
      "Epoch 42/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6940 - acc: 0.5353 - val_loss: 0.7197 - val_acc: 0.4308\n",
      "Epoch 43/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6939 - acc: 0.5353 - val_loss: 0.7196 - val_acc: 0.4308\n",
      "Epoch 44/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6939 - acc: 0.5353 - val_loss: 0.7195 - val_acc: 0.4308\n",
      "Epoch 45/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6939 - acc: 0.5353 - val_loss: 0.7195 - val_acc: 0.4308\n",
      "Epoch 46/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6939 - acc: 0.5353 - val_loss: 0.7194 - val_acc: 0.4308\n",
      "Epoch 47/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6939 - acc: 0.5353 - val_loss: 0.7193 - val_acc: 0.4308\n",
      "Epoch 48/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6939 - acc: 0.5353 - val_loss: 0.7192 - val_acc: 0.4308\n",
      "Epoch 49/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6938 - acc: 0.5353 - val_loss: 0.7192 - val_acc: 0.4308\n",
      "Epoch 50/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6938 - acc: 0.5353 - val_loss: 0.7191 - val_acc: 0.4308\n",
      "21135/22544 [===========================>..] - ETA: 0s\n",
      " Train Acc: 0.529719665646553, Test Acc: 0.43075762689113617\n",
      " \n",
      " Current Layer Attributes - epochs:50 hidden layers:2 features count:32\n",
      "Train on 112720 samples, validate on 22544 samples\n",
      "Epoch 1/50\n",
      "112720/112720 [==============================] - 3s - loss: 0.6936 - acc: 0.5353 - val_loss: 0.7190 - val_acc: 0.4308\n",
      "Epoch 2/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6931 - acc: 0.5353 - val_loss: 0.7148 - val_acc: 0.4308\n",
      "Epoch 3/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6929 - acc: 0.5353 - val_loss: 0.7143 - val_acc: 0.4308\n",
      "Epoch 4/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6928 - acc: 0.5353 - val_loss: 0.7141 - val_acc: 0.4308\n",
      "Epoch 5/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6927 - acc: 0.5353 - val_loss: 0.7137 - val_acc: 0.4308\n",
      "Epoch 6/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6926 - acc: 0.5353 - val_loss: 0.7134 - val_acc: 0.4308\n",
      "Epoch 7/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6926 - acc: 0.5353 - val_loss: 0.7130 - val_acc: 0.4308\n",
      "Epoch 8/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6925 - acc: 0.5353 - val_loss: 0.7130 - val_acc: 0.4308\n",
      "Epoch 9/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6925 - acc: 0.5353 - val_loss: 0.7127 - val_acc: 0.4308\n",
      "Epoch 10/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6924 - acc: 0.5353 - val_loss: 0.7127 - val_acc: 0.4308\n",
      "Epoch 11/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6924 - acc: 0.5353 - val_loss: 0.7124 - val_acc: 0.4308\n",
      "Epoch 12/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6923 - acc: 0.5353 - val_loss: 0.7122 - val_acc: 0.4308\n",
      "Epoch 13/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6923 - acc: 0.5353 - val_loss: 0.7121 - val_acc: 0.4308\n",
      "Epoch 14/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6923 - acc: 0.5353 - val_loss: 0.7120 - val_acc: 0.4308\n",
      "Epoch 15/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6922 - acc: 0.5353 - val_loss: 0.7120 - val_acc: 0.4308\n",
      "Epoch 16/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6922 - acc: 0.5353 - val_loss: 0.7119 - val_acc: 0.4308\n",
      "Epoch 17/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6922 - acc: 0.5353 - val_loss: 0.7117 - val_acc: 0.4308\n",
      "Epoch 18/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6922 - acc: 0.5353 - val_loss: 0.7117 - val_acc: 0.4308\n",
      "Epoch 19/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6921 - acc: 0.5353 - val_loss: 0.7116 - val_acc: 0.4308\n",
      "Epoch 20/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6921 - acc: 0.5353 - val_loss: 0.7115 - val_acc: 0.4308\n",
      "Epoch 21/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6921 - acc: 0.5353 - val_loss: 0.7115 - val_acc: 0.4308\n",
      "Epoch 22/50\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "112720/112720 [==============================] - 1s - loss: 0.6921 - acc: 0.5353 - val_loss: 0.7114 - val_acc: 0.4308\n",
      "Epoch 23/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6921 - acc: 0.5353 - val_loss: 0.7113 - val_acc: 0.4308\n",
      "Epoch 24/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6920 - acc: 0.5353 - val_loss: 0.7113 - val_acc: 0.4308\n",
      "Epoch 25/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6920 - acc: 0.5353 - val_loss: 0.7112 - val_acc: 0.4308\n",
      "Epoch 26/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6920 - acc: 0.5353 - val_loss: 0.7113 - val_acc: 0.4308\n",
      "Epoch 27/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6920 - acc: 0.5353 - val_loss: 0.7112 - val_acc: 0.4308\n",
      "Epoch 28/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6920 - acc: 0.5353 - val_loss: 0.7111 - val_acc: 0.4308\n",
      "Epoch 29/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6920 - acc: 0.5353 - val_loss: 0.7111 - val_acc: 0.4308\n",
      "Epoch 30/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6920 - acc: 0.5353 - val_loss: 0.7110 - val_acc: 0.4308\n",
      "Epoch 31/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6919 - acc: 0.5353 - val_loss: 0.7110 - val_acc: 0.4308\n",
      "Epoch 32/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6919 - acc: 0.5353 - val_loss: 0.7109 - val_acc: 0.4308\n",
      "Epoch 33/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6919 - acc: 0.5353 - val_loss: 0.7109 - val_acc: 0.4308\n",
      "Epoch 34/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6919 - acc: 0.5353 - val_loss: 0.7108 - val_acc: 0.4308\n",
      "Epoch 35/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6919 - acc: 0.5353 - val_loss: 0.7107 - val_acc: 0.4308\n",
      "Epoch 36/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6919 - acc: 0.5353 - val_loss: 0.7108 - val_acc: 0.4308\n",
      "Epoch 37/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6919 - acc: 0.5353 - val_loss: 0.7107 - val_acc: 0.4308\n",
      "Epoch 38/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6919 - acc: 0.5353 - val_loss: 0.7107 - val_acc: 0.4308\n",
      "Epoch 39/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6919 - acc: 0.5353 - val_loss: 0.7107 - val_acc: 0.4308\n",
      "Epoch 40/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6919 - acc: 0.5353 - val_loss: 0.7106 - val_acc: 0.4308\n",
      "Epoch 41/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6918 - acc: 0.5353 - val_loss: 0.7106 - val_acc: 0.4308\n",
      "Epoch 42/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6918 - acc: 0.5353 - val_loss: 0.7106 - val_acc: 0.4308\n",
      "Epoch 43/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6918 - acc: 0.5353 - val_loss: 0.7106 - val_acc: 0.4308\n",
      "Epoch 44/50\n",
      "112720/112720 [==============================] - 1s - loss: 0.6918 - acc: 0.5353 - val_loss: 0.7106 - val_acc: 0.4308\n",
      "Epoch 45/50\n",
      "  1409/112720 [..............................] - ETA: 2s - loss: 0.6897 - acc: 0.5500"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "#features_arr = [4, 16, 32, 256, 1024]\n",
    "#hidden_layers_arr = [2, 6, 10, 100]\n",
    "\n",
    "#features_arr = [4, 16, 32]\n",
    "#hidden_layers_arr = [2, 6, 10]\n",
    "\n",
    "features_arr = [4, 16, 32]\n",
    "hidden_layers_arr = [2, 4, 6]\n",
    "\n",
    "epoch_arr = [50]\n",
    "\n",
    "score = namedtuple(\"score\", ['epoch', 'no_of_features','hidden_layers','train_score', 'test_score'])\n",
    "scores = []\n",
    "predictions = {}\n",
    "\n",
    "for e, h, f in itertools.product(epoch_arr, hidden_layers_arr, features_arr):\n",
    "    \n",
    "    print(\" \\n Current Layer Attributes - epochs:{} hidden layers:{} features count:{}\".format(e,h,f))\n",
    "    latent_dim = f\n",
    "    epochs = e\n",
    "    hidden_layers = h\n",
    "    \n",
    "    train_size = x_train.shape[0] - x_train.shape[0]%batch_size\n",
    "    valid_size = x_valid.shape[0] - x_valid.shape[0]%batch_size\n",
    "\n",
    "    \n",
    "    optimizer = keras.optimizers.Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=1e-04, decay=0.1)\n",
    "    \n",
    "    vae_model = Model(inputs = Train.x, outputs = Train.y)\n",
    "    vae_model.compile(optimizer = optimizer, \n",
    "                      loss = keras.losses.categorical_crossentropy, metrics = ['accuracy'])   \n",
    "    \n",
    "    vae_model.fit(x = x_train[:train_size,:], y = y_train[:train_size,:], \n",
    "                  shuffle=True, epochs=epochs, \n",
    "                  batch_size = batch_size, \n",
    "                  validation_data = (x_test, y_test),\n",
    "                  verbose = 1)\n",
    "    score_train = vae_model.evaluate(x_valid[:valid_size,:], y = y_valid[:valid_size,:],\n",
    "                               batch_size = batch_size,\n",
    "                               verbose = 1)\n",
    "    \n",
    "    score_test = vae_model.evaluate(x_test, y = y_test,\n",
    "                           batch_size = batch_size,\n",
    "                           verbose = 1)\n",
    "    \n",
    "    y_test_pred = vae_model.predict(x_test, batch_size=batch_size)\n",
    "    \n",
    "\n",
    "    y_pred = y_test_pred #np.argmax(y_test_pred[:,-2:], axis = 1)\n",
    "    \n",
    "    curr_pred = pd.DataFrame({\"Attack_prob\":y_pred[:,0], \"Normal_prob\":y_pred[:,1]})\n",
    "    predictions.update({\"{}_{}_{}\".format(e,f,h):curr_pred})\n",
    "    \n",
    "    scores.append(score(e,f,h,score_train[-1], score_test[-1])) #score_test[-1]))\n",
    "    \n",
    "    print(\"\\n Train Acc: {}, Test Acc: {}\".format(score_train[-1], \n",
    "                                                  score_test[-1])  )\n",
    "    \n",
    "scores = pd.DataFrame(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2017-05-09T21:33:12.001Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "scores.sort_values(\"test_score\", ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2017-05-09T21:33:12.004Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.Panel(predictions).to_pickle(\"dataset/keras_vae_dense_trained_seperately_nsl_kdd_predictions.pkl\")\n",
    "scores.to_pickle(\"dataset/keras_vae_dense_trained_seperately_nsl_kdd_scores.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2017-05-09T21:33:12.007Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.Panel(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "_draft": {
   "nbviewer_url": "https://gist.github.com/33dcb1bcf3ca4a3461c4405a003a7591"
  },
  "anaconda-cloud": {},
  "gist": {
   "data": {
    "description": "Final Hyper parameter tuning",
    "public": false
   },
   "id": "33dcb1bcf3ca4a3461c4405a003a7591"
  },
  "kernelspec": {
   "display_name": "Python [conda env:p3]",
   "language": "python",
   "name": "conda-env-p3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
